import nltk
import pandas as pd
from gensim.utils import tokenize
from nltk import word_tokenize

nltk.download('stopwords')
nltk.download('punkt')
import re
from nltk.corpus import stopwords

# Reading the Dataset
# Reading the processed column dataset

df = pd.read_csv("Data/processed_numerical_columns.csv")
df 

# Extracting the only required columns and dropping the rest of the columns
df.columns
ndex(['Airport Name', 'Reviewer Name', 'Review Date', 'Reviewer Country',
       'Review Title', 'Review Rating', 'Review Verified Status',
       'Review Text', 'Experience At Airport', 'Date Visit',
       'Type Of Traveller', 'Queuing Times', 'Terminal Cleanliness',
       'Terminal Seating', 'Terminal Signs', 'Food Beverages',
       'Airport Shopping', 'Wifi Connectivity', 'Airport Staff',
       'Recommended'],
      dtype='object')
	  
	  df_subset = df[['Review Text', 'Queuing Times', 'Terminal Cleanliness', 'Terminal Seating', 'Terminal Signs', 
                'Food Beverages', 'Airport Shopping', 'Wifi Connectivity', 'Airport Staff', 'Recommended']]
df_subset

# Above dataset contains the label column (Recommended) and rest of the columns as features¶
# Pre-processing the features
# 1. Pre-processing Review Text Column

def _process_review(review):
    review = review.lower()
    review = list(word_tokenize(review))
    
    stop_words = set(stopwords.words('english')) 
    
    review = ' '.join([word for word in review if word not in stop_words])
    
    ## Removing Punctuation
    review = re.sub(r'[^\w\s]', '', review)
    
    review = ''.join([i for i in review if not i.isdigit()]).strip()
    return review
	
	def process_reviews(list_of_reviews):
    processed_reviews = []
    for review in list_of_reviews:
        processed_reviews.append(_process_review(review))
    return processed_reviews
	
	df_subset['processed_review_text'] = process_reviews(df_subset['Review Text'])
	df_subset
	
# Applying TF-IDF on the review column

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer()
x = tfidf.fit_transform(df_subset['processed_review_text'])

df_tfidf = pd.DataFrame(x.toarray(), columns=tfidf.get_feature_names())
df_tfidf

df_tfidf['Queuing Times'] = df_subset['Queuing Times']
df_tfidf['Terminal Cleanliness'] = df_subset['Terminal Cleanliness']
df_tfidf['Terminal Seating'] = df_subset['Terminal Seating']
df_tfidf['Terminal Signs'] = df_subset['Terminal Signs']
df_tfidf['Food Beverages'] = df_subset['Food Beverages']
df_tfidf['Airport Shopping'] = df_subset['Airport Shopping']
df_tfidf['Wifi Connectivity'] = df_subset['Wifi Connectivity']
df_tfidf['Airport Staff'] = df_subset['Airport Staff']
df_tfidf['Recommended'] = df_subset['Recommended']

df_tfidf


# Applying Logistic Regression

df_x = df_tfidf.drop('Recommended', axis=1)
df_y = df_tfidf[['Recommended']]

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.33, random_state=42)

from sklearn.linear_model import LogisticRegression

# all parameters not specified are set to their defaults
lr = LogisticRegression()

# Training the Model

lr.fit(X_train, y_train)
LogisticRegression()

# Predicting Using Model

y_test['Predictions'] = lr.predict(X_test)

# Model Evaluation

from sklearn.metrics import confusion_matrix

confusion_matrix(y_test['Recommended'], y_test['Predictions'])
array([[8701,  664],
       [ 942, 2268]], dtype=int64)
	   
	   from sklearn.metrics import classification_report

print(classification_report(y_test['Recommended'], y_test['Predictions']))

 precision    recall  f1-score   support

          no       0.90      0.93      0.92      9365
         yes       0.77      0.71      0.74      3210

    accuracy                           0.87     12575
   macro avg       0.84      0.82      0.83     12575
weighted avg       0.87      0.87      0.87     12575


